﻿using System;
using System.Collections.Generic;
using System.IO;
using System.Net;
using System.Threading;
using System.Xml;
using SDT.CrawlSystem.Servers.Filters;
using SDT.ServiceSystem;
using SDT.CrawlSystem.Servers;



namespace URLChannelTester
{
    /// <summary>
    /// Written by TungNX
    /// </summary>
    public sealed partial class UrlCrawler : WorkerThread
    {
       
        #region Properties from data file
        private string dataFile = null;
        public string DataFile
        {
            get { return dataFile; }
        }

       
        #endregion

        #region Constructors
        public UrlCrawler(string dataFile)
            : base()
        {
           
            this.dataFile = dataFile;

            LoadData();
        }
        #endregion

        private int CurrentWorkIndex = 0;
        private IDictionary<int, IList<UrlWorkUnit>> WorkUnitList = new Dictionary<int, IList<UrlWorkUnit>>();
       
        #region Load data
        private DateTime DataFileLastWriteTime = DateTime.MinValue;
        private bool MustReloadData()
        {
            var F = new FileInfo(dataFile);
            if (!F.Exists)
                return false;

            var ret = DataFileLastWriteTime != F.LastWriteTime;
            DataFileLastWriteTime = F.LastWriteTime;
            return ret;
        }

        private bool MustSaveData = false;

        private void LoadData()
        {
            if (MustReloadData())
            {
                #region Clear data before reloading
                ClearWork();
                CurrentWorkIndex = 0;
                WorkUnitList.Clear();
                #endregion

                #region Load data file into XmlDocument
                var d = new XmlDocument();
                var w = (new FileInfo(DataFile)).OpenText();
                var r = XmlReader.Create(w);
                //Modified Date : 03/10/2010
                //Author : TungNX
                //Try catch loi
                try
                {
                    d.Load(r);

                    r.Close();
                    w.Close();
                #endregion

                    #region Overrided properties
                    var n = d.SelectSingleNode("Channels/LogFile");
                    
                    n = d.SelectSingleNode("Channels/Interval");
                    
                    n = d.SelectSingleNode("Channels/WorkUnitInterval");
                    #endregion

                    
                    #region Load each channels
                    var channels = d.GetElementsByTagName("Channel");
                    foreach (var i in channels)
                        LoadData(i as XmlNode);
                    #endregion

                    #region Save data file
                    if (MustSaveData)
                        d.Save(DataFile);
                    MustSaveData = false;
                    #endregion

                    #region Prepare works list
                    foreach (var l in WorkUnitList.Values)
                        foreach (var work in l)
                            AddWork(work);
                    #endregion

                 
                }
                catch (Exception ex)
                {
                 
                    return;
                }
                finally
                {
                    r.Close();
                    w.Close();
                }
            }
        }

        /// <summary>
        /// Load channel's data
        /// </summary>
        private void LoadData(XmlNode n)
        {
            try
            {


                #region Channel's attributes
                var active = n.Attributes["Active"].Value == "1" ? true : false;
                if (!active)
                    return;
                var name = n.Attributes["Name"].Value;
                var id = Convert.ToInt32(n.Attributes["Id"].Value);
                if (id == 0)
                {
                    MustSaveData = true;
                }
                else if (n.Attributes["Update"].Value == "1")
                {
                    MustSaveData = true;
                    //update DB when id in file config <> 0
                }
                if (id == 0)
                    return;
                n.Attributes["Id"].Value = id.ToString();
                n.Attributes["Update"].Value = "0";
                #endregion

                foreach (var child in n.ChildNodes)
                    LoadData(id, child as XmlNode);
            }
            catch (Exception ex)
            {
                
            }
        }

        /// <summary>
        /// Load page's data
        /// </summary>
        private void LoadData(int cid, XmlNode p)
        {
           

            #region Page's attributes
            var name = p.Attributes["Name"].Value;
            var id = Convert.ToInt32(p.Attributes["Id"].Value);
            if (id == 0)
            {
                MustSaveData = true;
                     Logger.AddEntry(String.Format(" +add page: {0}[{1}]", name, id), 2);
            }
            else if (p.Attributes["Update"].Value == "1")
            {
                MustSaveData = true;
                  Logger.AddEntry(String.Format(" +upd page: {0}[{1}]", name, id), 2);
            }
            if (id == 0)
                return;
            p.Attributes["Id"].Value = id.ToString();
            p.Attributes["Update"].Value = "0";
            #endregion

            if (!WorkUnitList.ContainsKey(id))
                WorkUnitList.Add(id, new List<UrlWorkUnit>());
            #region Load region html . Vùng qui định lấy HTML
            //Author: TungNX
            //Created Date: 24/2/2011
            var hF = p.SelectSingleNode("HtmlFilter") != null ? new Filter(p.SelectSingleNode("HtmlFilter")) : null;

            #endregion

            #region Url Filter
            var n = p.SelectSingleNode("URLFilter");

            var min = Convert.ToInt32(n.Attributes["MinLen"].Value);
            var sta = n.Attributes["StartWith"].Value;
            var end = n.Attributes["EndWith"].Value;
            var con = n.Attributes["Contain"].Value;

            var ufilter = new UrlFilter(min, sta, end, con);
            #endregion

            #region WID Filter
            n = p.SelectSingleNode("WIDFilter");
            sta = n.Attributes["StartWith"].Value;
            end = n.Attributes["EndWith"].Value;
            //con = n.Attributes["RegEx"].Value;
            con = p.SelectSingleNode("WIDFilter").InnerText;

            var wFilter = new WIDFilter(0, sta, end, con);
            #endregion

            #region Urls Node
            n = p.SelectSingleNode("Urls");
            var pre = n.Attributes["Prefix"].Value;
            //wFilter.StartAt = pre.Length;
            #endregion

            foreach (var i in n.ChildNodes)
            {
                var l = Generator.GenUrls(pre, i as XmlNode);

                for (int j = 0; j < l.Count; j++)
                      WorkUnitList[id].Add(new UrlWorkUnit(cid, id,ufilter, wFilter,hF, pre, l[j], j));
                l.Clear();
            }
        }
        /// <summary>
        /// Author: TungNX
        /// Created Date: 23/2/2011
        /// Load file config to 
        /// </summary>
        /// <param name="p"></param>
        private void LoadRegionFilter(XmlNode n)
        {
            #region Load region html . Vùng qui định lấy HTML
            //Author: TungNX
            //Created Date: 24/2/2011
            var regHtml = n.SelectSingleNode("HtmlFilter") != null ? new Filter(n.SelectSingleNode("HtmlFilter")) : null;


            #endregion
        
        }
        #endregion




        public override void DoWork()
        {
            LoadData();
            var w = GetWork();
            if (w != null)
            {
                CurrentWorkIndex++;
                AddWork(w);

                if (w.Ready)
                    DownloadWebsite(w as UrlWorkUnit);
            }
        }

        #region Download webpage
        private class RequestState
        {
            // Holds the request
            public WebRequest Request;

            public UrlWorkUnit WorkUnit;

            public RequestState(WebRequest request, UrlWorkUnit workUnit)
            {
                Request = request;
                WorkUnit = workUnit;
            }
        }

        private CookieContainer cookie = null;
        public CookieContainer Cookie
        {
            get
            {
                if (cookie == null)
                    cookie = new CookieContainer();
                return cookie;
            }
        }

        private void DownloadWebsite(UrlWorkUnit w)
        {
            Logger.AddEntry(String.Format(" GET: {0}", w.Url));
            lock (w)
            {
                w.Ready = false;
                w.RawHtml = null;
            }

            try
            {
                var r = (HttpWebRequest)HttpWebRequest.Create(w.Url);
                r.UserAgent = "Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US; rv:1.9.1.8) Gecko/20100202 Firefox/3.5.8 GTB6 (.NET CLR 3.5.30729)";
                // Maintain same cookie
                r.CookieContainer = Cookie;
                r.Method = "GET";

                var state = new RequestState(r, w);
                var result = r.BeginGetResponse(new AsyncCallback(DownloadCompletedCallback), state);

                // Register the timeout callback
                ThreadPool.RegisterWaitForSingleObject(
                    result.AsyncWaitHandle, new WaitOrTimerCallback(DownloadTimeoutCallback), state, 30 * 1000, true
                );
            }
            catch (Exception ex)
            {
                Logger.AddEntry(String.Format(" EXC: {0} {1}", w.Url, ex.Message));
            }
        }

        private void DownloadCompletedCallback(IAsyncResult result)
        {
            try
            {
                var state = result.AsyncState as RequestState;
                var req = state.Request as WebRequest;

                // Get the Response
                var res = req.EndGetResponse(result) as HttpWebResponse;
                var s = res.GetResponseStream() as Stream;

                var r = new StreamReader(s);

                // Entire contents of the requested page
                lock (state.WorkUnit)
                {
                    state.WorkUnit.RawHtml = r.ReadToEnd();
                    state.WorkUnit.Ready = true;
                    if (ResultHandler != null)
                        ResultHandler(state.WorkUnit);
                    Logger.AddEntry(String.Format(" SUC: {0}", state.WorkUnit.Url));
                }

                res.Close();
                s.Close();
                r.Close();
            }
            catch (Exception ex)
            {
                Logger.AddEntry(String.Format(" EXC: {0} {1} {2}", (result.AsyncState as RequestState).WorkUnit.Url, ex, ex.Message));
            }
        }

        private void DownloadTimeoutCallback(object state, bool timedOut)
        {
            if (timedOut)
            {
                var r = state as RequestState;
                if (r != null)
                    r.Request.Abort();

                lock (r.WorkUnit)
                {
                    r.WorkUnit.RawHtml = null;
                    r.WorkUnit.Ready = true;
                    if (ResultHandler != null)
                        ResultHandler(r.WorkUnit);
                    Logger.AddEntry(String.Format(" TIMEUP: {0}", r.WorkUnit.Url));
                }
            }
        }

        #endregion

        #region Result handler
        private static bool SaveResult(IWorkUnit w)
        {
            if (w == null || !w.Ready || (w as UrlWorkUnit).RawHtml == null)
                return false;

            (w as UrlWorkUnit).SaveResult();

            return true;
        }

        public override WorkResultHandlerDelegate ResultHandler
        {
            protected get
            {
                if (base.ResultHandler == null)
                    ResultHandler = new WorkResultHandlerDelegate(SaveResult);
                return base.ResultHandler;
            }
            set
            {
                base.ResultHandler = value;
            }
        }
        #endregion
    }
}